union Bin = NominalBin | IntervalBin | MissingValueBin

type Cluster {
  """The ID of the cluster"""
  id: ID!

  """The event IDs of the points in the cluster"""
  eventIds: [ID!]!

  """Ratio of primary points over reference points"""
  driftRatio: Float

  """Ratio of primary points over corpus points"""
  primaryToCorpusRatio: Float

  """
  Data quality metric summarized by the respective datasets of the clustered events
  """
  dataQualityMetric(metric: DataQualityMetricInput!): DatasetValues!

  """
  Performance metric summarized by the respective datasets of the clustered events
  """
  performanceMetric(metric: PerformanceMetricInput!): DatasetValues!
}

input ClusterInput {
  eventIds: [ID!]!
  id: ID
}

enum DataQualityMetric {
  cardinality
  percentEmpty
  mean
  sum
  min
  max
  count
  p01
  p25
  p50
  p75
  p99
}

input DataQualityMetricInput {
  metric: DataQualityMetric!
  columnName: String
}

type DataQualityTimeSeries implements TimeSeries {
  data: [TimeSeriesDataPoint!]!
}

type Dataset {
  """The start bookend of the data"""
  startTime: DateTime!

  """The end bookend of the data"""
  endTime: DateTime!

  """The record count of the data"""
  recordCount: Int!

  """Returns a human friendly name for the dataset."""
  name: String!
  events(eventIds: [ID!]!, dimensions: [DimensionInput!]): [Event!]!
}

enum DatasetRole {
  primary
  reference
}

type DatasetValues {
  primaryValue: Float
  referenceValue: Float
}

"""Date with time (isoformat)"""
scalar DateTime

type Dimension implements Node {
  id: GlobalID!

  """The name of the dimension (a.k.a. the column name)"""
  name: String!

  """
  Whether the dimension represents a feature, tag, prediction, or actual.
  """
  type: DimensionType!

  """The data type of the column. Categorical or numeric."""
  dataType: DimensionDataType!

  """Whether the dimension data is continuous or discrete."""
  shape: DimensionShape!
  driftMetric(metric: ScalarDriftMetric!, timeRange: TimeRange): Float
  dataQualityMetric(
    metric: DataQualityMetric!
    timeRange: TimeRange

    """The dataset (primary or reference) to query"""
    datasetRole: DatasetRole = primary
  ): Float

  """
  Returns the observed categories of a categorical dimension (usually a dimension of string values) as a list of unique string labels sorted in lexicographical order. Missing values are excluded. Non-categorical dimensions return an empty list.
  """
  categories: [String!]!

  """
  Returns the time series of the specified metric for data within a time range. Data points are generated starting at the end time and are separated by the sampling interval. Each data point is labeled by the end instant and contains data from their respective evaluation windows.
  """
  dataQualityTimeSeries(
    metric: DataQualityMetric!
    timeRange: TimeRange!
    granularity: Granularity!

    """The dataset (primary or reference) to query"""
    datasetRole: DatasetRole = primary
  ): DataQualityTimeSeries!

  """
  The time series of the specified metric for data within a time range. Data points are generated starting at the end time and are separated by the sampling interval. Each data point is labeled by the end instant and contains data from their respective evaluation windows.
  """
  driftTimeSeries(metric: ScalarDriftMetric!, timeRange: TimeRange!, granularity: Granularity!): DriftTimeSeries!

  """
  Returns the segments across both datasets and returns the counts per segment
  """
  segmentsComparison(primaryTimeRange: TimeRange): Segments!
}

type DimensionConnection {
  pageInfo: PageInfo!
  edges: [DimensionEdge!]!
}

enum DimensionDataType {
  categorical
  numeric
}

type DimensionEdge {
  node: Dimension!
  cursor: String!
}

input DimensionFilter {
  types: [DimensionType!]
  shapes: [DimensionShape!]
  dataTypes: [DimensionDataType!]
}

input DimensionInput {
  name: String!
  type: DimensionType!
}

enum DimensionShape {
  continuous
  discrete
}

enum DimensionType {
  feature
  tag
  prediction
  actual
}

type DimensionWithValue {
  dimension: Dimension!

  """The string representation of the dimension's value"""
  value: String
}

type DocumentEvaluation implements Evaluation {
  """Name of the evaluation, e.g. 'helpfulness' or 'relevance'."""
  name: String!

  """Result of the evaluation in the form of a numeric score."""
  score: Float

  """
  Result of the evaluation in the form of a string, e.g. 'helpful' or 'not helpful'. Note that the label is not necessarily binary.
  """
  label: String

  """
  The evaluator's explanation for the evaluation result (i.e. score or label, or both) given to the subject.
  """
  explanation: String

  """
  The zero-based index among retrieved documents, which is collected as a list (even when ordering is not inherently meaningful).
  """
  documentPosition: Int!
}

"""
Summarization of retrieval metrics: Average NDCG@K, Average Precision@K, Mean Reciprocal Rank, Hit Rate, etc.
"""
type DocumentEvaluationSummary {
  evaluationName: String!
  averageNdcg(k: Int): Float
  countNdcg(k: Int): Int!
  averagePrecision(k: Int): Float
  countPrecision(k: Int): Int!
  meanReciprocalRank: Float
  countReciprocalRank: Int!
  hitRate: Float
  countHit: Int!
}

"""
A collection of retrieval metrics computed on a list of document evaluation scores: NDCG@K, Precision@K, Reciprocal Rank, etc.
"""
type DocumentRetrievalMetrics {
  evaluationName: String!

  """
  Normalized Discounted Cumulative Gain (NDCG) at `k` with log base 2 discounting. If `k` is None, it's set to the length of the scores. If `k` < 1, return 0.0.
  """
  ndcg(k: Int): Float

  """
  Precision at `k`, defined as the fraction of truthy scores among first `k` positions (1-based index). If `k` is None, then it's set to the length of the scores. If `k` < 1, return 0.0.
  """
  precision(k: Int): Float

  """
  Return `1/R` where `R` is the rank of the first hit, i.e. the 1-based index position of first truthy score, e.g. score=1. If a non-finite value (e.g. `NaN`) is encountered before the first (finite) truthy score, then return `NaN`, otherwise if no truthy score is found (or if the count of scores is zero), return 0.0.
  """
  reciprocalRank: Float

  """
  Return 1.0 if any score is truthy (i.e. is a hit), e.g. score=1. Otherwise, return `NaN` if any score is non-finite (e.g. `NaN`), or return 0.0 if all scores are falsy, e.g. all scores are 0.
  """
  hit: Float
}

type DriftTimeSeries implements TimeSeries {
  data: [TimeSeriesDataPoint!]!
}

type EmbeddingDimension implements Node {
  id: GlobalID!
  name: String!

  """
  Computes a drift metric between all reference data and the primary data belonging to the input time range (inclusive of the time range start and exclusive of the time range end). Returns None if no reference dataset exists, if no primary data exists in the input time range, or if the input time range is invalid.
  """
  driftMetric(metric: VectorDriftMetric!, timeRange: TimeRange): Float

  """
  Computes a retrieval metric between corpus data and the primary data belonging to the input time range (inclusive of the time range start and exclusive of the time range end). Returns None if no reference dataset exists, if no primary data exists in the input time range, or if the input time range is invalid.
  """
  retrievalMetric(metric: VectorDriftMetric!, timeRange: TimeRange): Float

  """
  Returns the time series of the specified metric for data within timeRange. Data points are generated starting at the end time, are separated by the sampling interval. Each data point is labeled by the end instant of and contains data from their respective evaluation window.
  """
  dataQualityTimeSeries(
    metric: DataQualityMetric!
    timeRange: TimeRange!
    granularity: Granularity!

    """The dataset (primary or reference) to query"""
    datasetRole: DatasetRole = primary
  ): DataQualityTimeSeries!

  """
  Computes a drift time-series between the primary and reference datasets. The output drift time-series contains one data point for each whole hour in the input time range (inclusive of the time range start and exclusive of the time range end). Each data point contains the drift metric value between all reference data and the primary data within the evaluation window ending at the corresponding time. Returns None if no reference dataset exists or if the input time range is invalid.           
  """
  driftTimeSeries(metric: VectorDriftMetric!, timeRange: TimeRange!, granularity: Granularity!): DriftTimeSeries!

  """
  Computes a retrieval metric between the primary and corpus datasets. The output time-series contains one data point for each whole hour in the input time range (inclusive of the time range start and exclusive of the time range end). Each data point contains the metric value between all corpus data and the primary data within the evaluation window ending at the corresponding time. Returns None if no corpus dataset exists or if the input time range is invalid.           
  """
  retrievalMetricTimeSeries(metric: VectorDriftMetric!, timeRange: TimeRange!, granularity: Granularity!): DriftTimeSeries!
  UMAPPoints(
    """The time range of the primary dataset to generate the UMAP points for"""
    timeRange: TimeRange!

    """UMAP target dimension hyperparameter. Must be 2 or 3"""
    nComponents: Int = 3

    """UMAP minimum distance hyperparameter"""
    minDist: Float! = 0

    """UMAP N neighbors hyperparameter"""
    nNeighbors: Int! = 30

    """UMAP N samples"""
    nSamples: Int! = 500

    """HDBSCAN minimum cluster size"""
    minClusterSize: Int! = 10

    """HDBSCAN minimum samples"""
    clusterMinSamples: Int! = 1

    """HDBSCAN cluster selection epsilon"""
    clusterSelectionEpsilon: Float! = 0
  ): UMAPPoints!
}

type EmbeddingDimensionConnection {
  pageInfo: PageInfo!
  edges: [EmbeddingDimensionEdge!]!
}

type EmbeddingDimensionEdge {
  node: EmbeddingDimension!
  cursor: String!
}

type EmbeddingMetadata {
  predictionId: String
  rawData: String
  linkToData: String
}

enum EvalAttr {
  score
  label
}

input EvalResultKey {
  name: String!
  attr: EvalAttr!
}

interface Evaluation {
  """Name of the evaluation, e.g. 'helpfulness' or 'relevance'."""
  name: String!

  """Result of the evaluation in the form of a numeric score."""
  score: Float

  """
  Result of the evaluation in the form of a string, e.g. 'helpful' or 'not helpful'. Note that the label is not necessarily binary.
  """
  label: String

  """
  The evaluator's explanation for the evaluation result (i.e. score or label, or both) given to the subject.
  """
  explanation: String
}

type EvaluationSummary {
  count: Int!
  labels: [String!]!
  labelFractions: [LabelFraction!]!
  meanScore: Float
  scoreCount: Int!
  labelCount: Int!
}

type Event {
  id: ID!
  eventMetadata: EventMetadata!
  dimensions: [DimensionWithValue!]!

  """The prompt and response pair associated with the event"""
  promptAndResponse: PromptResponse

  """The text of the document if the event is a retrieved document record"""
  documentText: String
}

type EventMetadata {
  predictionId: String
  predictionScore: Float
  predictionLabel: String
  actualScore: Float
  actualLabel: String
}

type ExportedFile {
  """File name without the file extension."""
  fileName: String!
}

type Functionality {
  """Model inferences are available for analysis"""
  modelInferences: Boolean!

  """Generative tracing records are available for analysis"""
  tracing: Boolean!
}

"""
The `ID` scalar type represents a unique identifier, often used to refetch an object or as key for a cache. The ID type appears in a JSON response as a String; however, it is not intended to be human-readable. When expected as an input type, any string (such as `"4"`) or integer (such as `4`) input value will be accepted as an ID.
"""
scalar GlobalID @specifiedBy(url: "https://relay.dev/graphql/objectidentification.htm")

"""
Granularity specifies the distance between points in a time-series and the duration of time (i.e. evaluation window) by which data is aggregated for  each data point. By convention all time intervals are right-open intervals, i.e. the end instant of the evaluation window is excluded from the interval. As a matter of standardization, each point in a time-series aggregates data  corresponding to an interval of time (i.e. the evaluation window) ending at the point's timestamp, and each time-series enumerates its points starting from the end instant of the TimeRange.
"""
input Granularity {
  """
  Specifies the length of time by which the data are grouped for aggregation. Each point in a time-series will have the same evaluation_window, but the evaluation_window for each point can overlap in real time. For example, when the points are 24 hours apart but the eval window is 72 hours, it means that each point in the time-series is aggregating 72 hours worth of data ending at the point's timestamp.
  """
  evaluationWindowMinutes: Int!

  """
  Specifies the time interval between each point in the time-series. All points in the time-series are separated by the same length of time, and are generated starting from the end time of the time range.
  """
  samplingIntervalMinutes: Int!
}

input InputCoordinate2D {
  x: Float!
  y: Float!
}

input InputCoordinate3D {
  x: Float!
  y: Float!
  z: Float!
}

type IntervalBin {
  range: NumericRange!
}

type LabelFraction {
  label: String!
  fraction: Float!
}

enum MimeType {
  text
  json
}

type MissingValueBin {
  name: String
}

type Model {
  dimensions(first: Int = 50, last: Int, after: String, before: String, include: DimensionFilter, exclude: DimensionFilter): DimensionConnection!
  primaryDataset: Dataset!
  referenceDataset: Dataset
  corpusDataset: Dataset
  embeddingDimensions(first: Int = 50, last: Int, after: String, before: String): EmbeddingDimensionConnection!

  """Returns exported file names sorted by descending modification time."""
  exportedFiles: [ExportedFile!]!
  performanceMetric(
    metric: PerformanceMetricInput!
    timeRange: TimeRange

    """The dataset (primary or reference) to query"""
    datasetRole: DatasetRole = primary
  ): Float

  """
  Returns the time series of the specified metric for data within a time range. Data points are generated starting at the end time and are separated by the sampling interval. Each data point is labeled by the end instant and contains data from their respective evaluation windows.
  """
  performanceTimeSeries(
    metric: PerformanceMetricInput!
    timeRange: TimeRange!
    granularity: Granularity!

    """The dataset (primary or reference) to query"""
    datasetRole: DatasetRole = primary
  ): PerformanceTimeSeries!
}

type Mutation {
  """
  Given a list of event ids, export the corresponding data subset in Parquet format. File name is optional, but if specified, should be without file extension. By default the exported file name is current timestamp.
  """
  exportEvents(eventIds: [ID!]!, fileName: String): ExportedFile!

  """
  Given a list of clusters, export the corresponding data subset in Parquet format. File name is optional, but if specified, should be without file extension. By default the exported file name is current timestamp.
  """
  exportClusters(clusters: [ClusterInput!]!, fileName: String): ExportedFile!
}

"""A node in the graph with a globally unique ID"""
interface Node {
  id: GlobalID!
}

type NominalBin {
  name: String!
}

type NumericRange {
  start: Float!
  end: Float!
}

type PageInfo {
  hasNextPage: Boolean!
  hasPreviousPage: Boolean!
  startCursor: String
  endCursor: String
  totalCount: Int!
}

enum PerformanceMetric {
  accuracyScore
}

input PerformanceMetricInput {
  metric: PerformanceMetric!
}

type PerformanceTimeSeries implements TimeSeries {
  data: [TimeSeriesDataPoint!]!
}

type Point2D {
  x: Float!
  y: Float!
}

union Point2DPoint3D = Point2D | Point3D

type Point3D {
  x: Float!
  y: Float!
  z: Float!
}

type Project implements Node {
  id: GlobalID!
  name: String!
  startTime: DateTime
  endTime: DateTime
  recordCount(timeRange: TimeRange): Int!
  traceCount(timeRange: TimeRange): Int!
  tokenCountTotal: Int!
  latencyMsP50: Float
  latencyMsP99: Float
  spans(timeRange: TimeRange, traceIds: [ID!], first: Int = 50, last: Int, after: String, before: String, sort: SpanSort, rootSpansOnly: Boolean, filterCondition: String): SpanConnection!

  """
  Names of all available evaluations for spans. (The list contains no duplicates.)
  """
  spanEvaluationNames: [String!]!

  """Names of available document evaluations."""
  documentEvaluationNames(spanId: ID): [String!]!
  spanEvaluationSummary(evaluationName: String!, timeRange: TimeRange, filterCondition: String): EvaluationSummary
  documentEvaluationSummary(evaluationName: String!, timeRange: TimeRange, filterCondition: String): DocumentEvaluationSummary
  streamingLastUpdatedAt: DateTime
  validateSpanFilterCondition(condition: String!): ValidationResult!
}

type ProjectConnection {
  pageInfo: PageInfo!
  edges: [ProjectEdge!]!
}

type ProjectEdge {
  node: Project!
  cursor: String!
}

type PromptResponse {
  """The prompt submitted to the LLM"""
  prompt: String

  """The response generated by the LLM"""
  response: String
}

type Query {
  projects(first: Int = 50, last: Int, after: String, before: String): ProjectConnection!
  functionality: Functionality!
  model: Model!
  node(id: GlobalID!): Node!
  clusters(clusters: [ClusterInput!]!): [Cluster!]!
  hdbscanClustering(
    """Event ID of the coordinates"""
    eventIds: [ID!]!

    """Point coordinates. Must be either 2D or 3D."""
    coordinates2d: [InputCoordinate2D!]

    """Point coordinates. Must be either 2D or 3D."""
    coordinates3d: [InputCoordinate3D!]

    """HDBSCAN minimum cluster size"""
    minClusterSize: Int! = 10

    """HDBSCAN minimum samples"""
    clusterMinSamples: Int! = 1

    """HDBSCAN cluster selection epsilon"""
    clusterSelectionEpsilon: Float! = 0
  ): [Cluster!]!
}

type Retrieval {
  queryId: ID!
  documentId: ID!
  relevance: Float
}

enum ScalarDriftMetric {
  psi
  klDivergence
  jsDistance
}

type Segment {
  bin: Bin!
  counts: DatasetValues!
}

type Segments {
  segments: [Segment!]!
  totalCounts: DatasetValues!
}

enum SortDir {
  asc
  desc
}

type Span {
  name: String!
  statusCode: SpanStatusCode!
  statusMessage: String!
  startTime: DateTime!
  endTime: DateTime
  latencyMs: Float

  """the parent span ID. If null, it is a root span"""
  parentId: ID
  spanKind: SpanKind!
  context: SpanContext!

  """Span attributes as a JSON string"""
  attributes: String!

  """Metadata as a JSON string"""
  metadata: String
  numDocuments: Int
  tokenCountTotal: Int
  tokenCountPrompt: Int
  tokenCountCompletion: Int
  input: SpanIOValue
  output: SpanIOValue
  events: [SpanEvent!]!

  """
  Cumulative (prompt plus completion) token count from self and all descendant spans (children, grandchildren, etc.)
  """
  cumulativeTokenCountTotal: Int

  """
  Cumulative (prompt) token count from self and all descendant spans (children, grandchildren, etc.)
  """
  cumulativeTokenCountPrompt: Int

  """
  Cumulative (completion) token count from self and all descendant spans (children, grandchildren, etc.)
  """
  cumulativeTokenCountCompletion: Int

  """
  Propagated status code that percolates up error status codes from descendant spans (children, grandchildren, etc.)
  """
  propagatedStatusCode: SpanStatusCode!

  """
  Evaluations associated with the span, e.g. if the span is an LLM, an evaluation may assess the helpfulness of its response with respect to its input.
  """
  spanEvaluations: [SpanEvaluation!]!

  """
  Evaluations of the documents associated with the span, e.g. if the span is a RETRIEVER with a list of documents in its RETRIEVAL_DOCUMENTS attribute, an evaluation for each document may assess its relevance respect to the input query of the span. Note that RETRIEVAL_DOCUMENTS is a list, and each evaluation is identified by its document's (zero-based) index in that list.
  """
  documentEvaluations: [DocumentEvaluation!]!

  """Retrieval metrics: NDCG@K, Precision@K, Reciprocal Rank, etc."""
  documentRetrievalMetrics(evaluationName: String): [DocumentRetrievalMetrics!]!

  """All descendant spans (children, grandchildren, etc.)"""
  descendants: [Span!]!
}

enum SpanColumn {
  startTime
  endTime
  latencyMs
  tokenCountTotal
  tokenCountPrompt
  tokenCountCompletion
  cumulativeTokenCountTotal
  cumulativeTokenCountPrompt
  cumulativeTokenCountCompletion
}

type SpanConnection {
  pageInfo: PageInfo!
  edges: [SpanEdge!]!
}

type SpanContext {
  traceId: ID!
  spanId: ID!
}

type SpanEdge {
  node: Span!
  cursor: String!
}

type SpanEvaluation implements Evaluation {
  """Name of the evaluation, e.g. 'helpfulness' or 'relevance'."""
  name: String!

  """Result of the evaluation in the form of a numeric score."""
  score: Float

  """
  Result of the evaluation in the form of a string, e.g. 'helpful' or 'not helpful'. Note that the label is not necessarily binary.
  """
  label: String

  """
  The evaluator's explanation for the evaluation result (i.e. score or label, or both) given to the subject.
  """
  explanation: String
}

type SpanEvent {
  name: String!
  message: String!
  timestamp: DateTime!
}

type SpanIOValue {
  mimeType: MimeType!
  value: String!
}

enum SpanKind {
  chain
  tool
  llm
  retriever
  embedding
  agent
  reranker
  unknown
}

"""
The sort key and direction for span connections. Must specify one and only one of either `col` or `evalResultKey`.
"""
input SpanSort {
  col: SpanColumn
  evalResultKey: EvalResultKey
  dir: SortDir!
}

enum SpanStatusCode {
  OK
  ERROR
  UNSET
}

input TimeRange {
  """The start of the time range"""
  start: DateTime!

  """The end of the time range. Right exclusive."""
  end: DateTime!
}

interface TimeSeries {
  data: [TimeSeriesDataPoint!]!
}

type TimeSeriesDataPoint {
  timestamp: DateTime!
  value: Float
}

type UMAPPoint {
  id: GlobalID!

  """The ID of the event that the point is a projection of"""
  eventId: ID!
  coordinates: Point2DPoint3D!
  embeddingMetadata: EmbeddingMetadata!
  eventMetadata: EventMetadata!
}

type UMAPPoints {
  data: [UMAPPoint!]!
  referenceData: [UMAPPoint!]!
  clusters: [Cluster!]!
  corpusData: [UMAPPoint!]!
  contextRetrievals: [Retrieval!]!
}

type ValidationResult {
  isValid: Boolean!
  errorMessage: String
}

enum VectorDriftMetric {
  euclideanDistance
}
